#! /usr/bin/python
#
# $Id$
#
# RTEMS Project (http://www.rtems.org/)
# Copyright 2007 Chris Johns (chrisj@rtems.org)
# All rights reserved.
#
# This file is part of The RTEMS Tools.
#
# The The RTEMS Tools is free software; you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation; either version 2 of the License, or
# (at your option) any later version.
#
# The The RTEMS Tools is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.
#
# You should have received a copy of the GNU General Public License
# along with The RTEMS Tools; if not, write to the Free Software
# Foundation, Inc., 51 Franklin St, Fifth Floor, Boston, MA  02110-1301  USA
#
# The Symbols tool extracts the symbols from the debug information of an
# object file or library.
#

import fileinput
import fnmatch
import optparse
import os
import re
import sys

import config
import environ
import error
import execute
import command
import log

#
# The version and copyright notice for this tool.
#
version = '0.0.1'
title = 'RTEMS Symbols Tool Version ' + version
copyright = (title,
             'Copyright 2007 Chris Johns (chrisj@rtems.org)',
             'The RTEMS Tools is free software, covered by the GNU General',
             'Public License, and you are welcome to change it and/or distribute',
             'copies of it under certain conditions. There is absolutely no',
             'warranty for the The RTEMS Tools.')

#
# Command line options.
#
cl_options = [
    optparse.make_option('-V', action = 'store_true',
                         dest = 'version',
                         default = False,
                         help='Show the RTEMS Symbols version.'),
    optparse.make_option('-v', action = 'store_true',
                         dest = 'verbose',
                         default = False,
                         help='Verbose messages and tracing.'),
    optparse.make_option('-q', action = 'store_true',
                         dest = 'quiet',
                         default = False,
                         help='Quiet showing only the source files'),
    optparse.make_option('-c', action = 'store', type = 'string',
                         dest = 'config',
                         default = 'config/symbols.cf',
                         help = 'Set the RTEMS Symbols configuration file.'),
    optparse.make_option('-F', action = 'store_true',
                         dest = 'showfunctions',
                         default = False,
                         help = 'Show the functions.'),
    optparse.make_option('-S', action = 'store_true',
                         dest = 'showstructs',
                         default = False,
                         help = 'Show the structures.'),
    optparse.make_option('-T', action = 'store_true',
                         dest = 'showtypes',
                         default = False,
                         help = 'Show the types.'),
    optparse.make_option('-U', action = 'store_true',
                         dest = 'showunions',
                         default = False,
                         help = 'Show the unions.'),
    optparse.make_option('-A', action = 'store_true',
                         dest = 'showall',
                         default = False,
                         help='Show all symbols when show differences.'),
    optparse.make_option('-R', action = 'store_true',
                         dest = 'rawdata',
                         default = True,
                         help = 'The data file is the objdump data.'),
    optparse.make_option('-o', action = 'store', type = 'string',
                         dest = 'objectfile',
                         default = 'test/symbols-data.txt',
                         help = 'The object file to process.'),
    optparse.make_option('-f', action = 'store', type = 'string',
                         dest = 'functions',
                         default = 'test/functions.txt',
                         help = 'The list of functions to process.')
]

#
# Verbose is global.
#
verbose = False

#
# Executed from the command line.
#
command_line = __name__ == "__main__"

#
# The machines to architectures mappings
#
machines = { 'MC68000' : 'm68k',
             'Intel 80386' : 'i386' }

#
# List of tags that are ignored.
#
ignore_tags = [
    '!_TAG_FILE_FORMAT',
    '!_TAG_FILE_SORTED',
    '!_TAG_PROGRAM_AUTHOR',
    '!_TAG_PROGRAM_NAME',
    '!_README'         # To write readme info for demo file
]

#
# Kind label to table keys.
#
table_keys = {
    't': 'typedef',
    'e': 'enum',
    's': 'struct',
    'u': 'union',
    'm': 'member',
    'v': 'variable',
    'g': 'global',        # guessed this name
    'f': 'function',
    'x': 'class-static-member',
    'p': 'class-member',
    'c': 'class-end-class'    
    }

#
# The types that have members.
#
member_types = ( table_keys['e'], table_keys['s'], table_keys['u'] )

#
# The list of keys to ignore.
#
table_keys_ignore = ( 'v', 'x', 'p', 'c' )

def output(text = None, l = None):
    """Output text if run from the command line. If not run from the
    command line or quite do not print the text."""
    if command_line:
        log.output(text, l)

class symbols:
    def __init__(self, cfgfile, verbose = False):
        self.verbose = verbose
        self.arch = 'none'
        self.config = None
        self.env = None
        self.exe = None
        # Read the configuration data.
        self.config = config.data()
        self.config.load(cfgfile)
        if self.verbose:
            output(str(self.config))
        # Get the local environment.
        self.env = environ.system(self.config, 'funcdecl', self.verbose)
        # Get the RTEMS version.
        rv = self.config.get_member('funcdecl', 'rtems_version')
        if len(rv) == 0:
            raise error.general('cannot locate the RTEMS version')
        self.rtems_version = rv[0][1]
        # Execute object to run command with.
        self.exe = execute.execute(output = output, verbose = verbose)
        self.all_functions = True
        self.functions = {}
        self.table = {}
        for k in table_keys:
            if k not in table_keys_ignore:
                self.table[table_keys[k]] = {}
        self.table['unknown'] = {}
        self.anon = {}
        self.anon_count = 0
        self.line = 0
        self.types = []

    def __del__(self):
        del self.exe
        del self.env
        del self.config
        
    def _is_enum(self, definition):
        return definition.startswith('enum')
    
    def _transform_enum(self, definition):
        """Transform the enumeration definition into the enum label."""
        definition = definition.strip()
        if self._is_enum(definition):
            if definition.find('{') >= 0:
                e = definition.split('}', 1)
                if len(e) <= 1:
                    raise error.general('invalid enum format: %s' % (definition))
                e[0] += '}'
                if e[0] in self.table['reverse']['enum']:
                    return self.table['reverse']['enum'][e[0]] + e[1]
                print self.table['reverse']
                raise error.general('enum not found: %s' % (e[0]))
        return definition

    def _readelf_output(self, line):
        """Output capture handler for the nm process."""
        if self.arch == 'none':
            sl = line.strip().split(' ', 1)
            if sl[0] == 'Machine:':
                what = sl[1].strip()
                if what in machines:
                    self.arch = machines[what]
                    
    def _parse_debug_rec(self, rec):
        """Extract the debug field from the debug record.
        Split in half on the ';"' delimiter. The first half as the name
        and file and line number, and the second half as a range of
        fields, each with a field label."""
        fields = {}
        f1 = rec.strip().split(';"', 1)
        if len(f1) != 2:
            raise error.general('invalid debug record format (;"): ' + rec)
        fl1 = f1[0].split('\t')
        if len(fl1) != 3:
            raise error.general('invalid debug record format: p1: ' + f1[0] + rec)
        fl2 = f1[1].split('\t')
        fields['name'] = fl1[0].strip()
        fields['file'] = fl1[1].strip()
        for f in fl2:
            f = f.strip()
            if len(f):
                fl3 = f.split(':', 1)
                if len(fl3) != 2:
                    raise error.general('invalid debug record format: p3: ' +
                                        f + ': ' + rec)
                fields[fl3[0]] = fl3[1].strip()
        return fields

    def _is_anon(self, rec):
        """Does the record contain an '%anon' field."""
        for f in rec:
            if rec[f].startswith('%anon'):
                return True;
        return False

    def _anon_extract(self, field):
        fs = field
        i = field.find('%anon') 
        if i >= 0:
            fs = fs[i:]
            i = fs.find(' ')
            if i > 0:
                fs = fs[:i]
            i = fs.find('(')
            if i > 0:
                fs = fs[i:]
            i = fs.find(')')
            if i > 0:
                fs = fs[:i]
            i = fs.find(',')
            if i > 0:
                fs = fs[:i]
            if fs.startswith('%anon'):
                return fs
            else:
                raise error.general('lost anon: ' + fs)
        return None
        

    def _anon_rename_field(self, field):
        """Find the current anon name and replace with the new anon name."""
        anon = self._anon_extract(field)
        if anon:
            if anon in self.anon:
                field = field.replace(anon, self.anon[anon])
            else:
                output('warning: anon name not found: ' + field +
                       ' (' + anon + ')')
                raise error.general('ouch')
        return field

    def _anon_rename(self, rec):
        """Find the current anon name and replace with the new anon name."""
        for f in rec:
            if type(rec[f]) is str:
                rec[f] = self._anon_rename_field(rec[f])
            elif type(rec[f]) is list:
                for i in range(0, len(rec[f])):
                    rec[f][i] = self._anon_rename_field(rec[f][i])
        return rec

    def _objdump_output(self, line):
        """Output capture handler for the objdump process."""

        global verbose

        if verbose and False:
            log.output(line)
        self.line += 1
        if line.find(';"') != -1:
            sl = line.strip().split('\t', 1)
            name = sl[0]
            if name not in ignore_tags:
                rec = self._parse_debug_rec(line)
                if 'kind' not in rec:
                    raise error.general('no kind found in debug record: ' + line)
                if 'name' not in rec:
                    raise error.general('no name found in debug record: ' + line)
                kind = rec['kind']
                name = rec['name']
                # If a name starts with %anon it is an anonomous items as described
                # below. These are a special problem because the names can be repeated
                # in files so we can get more than one %anon1 for different items.
                # As a result we need to rename them.
                if name.startswith('%anon'):
                    anon = name + ('_%d' % (self.anon_count))
                    self.anon[name] = anon
                    self.anon_count += 1
                    name = anon
                    if name == '%anon1_4594':
                        verbose = False
                if verbose:
                    output('INPUT  :' + self.str_rec(rec))
                rec = self._anon_rename(rec)
                if verbose:
                    output('RENAMED:' + self.str_rec(rec))
                if kind in table_keys:
                    key = table_keys[kind]
                    # If a member of a structure, union or enum we need
                    # find the structure, union or enum and assign the
                    # members to it. The order is important.
                    #
                    # For example consider the following struct out of the
                    # standard header 'sys/signal.h;:
                    #
                    # typedef struct {
                    #   int          si_signo;    /* Signal number */
                    #   int          si_code;     /* Cause of the signal */
                    #   union sigval si_value;    /* Signal value */
                    # } siginfo_t;
                    #
                    # Gives:
                    #
                    # kind:s name:%anon64, file: sys/signal.h
                    # kind:m name:si_signo, access: public, file: sys/signal.h, \
                    #                       type: int, struct: %anon64
                    # kind:m name:si_code, access: public, file: sys/signal.h, \
                    #                      type: int, struct: %anon64
                    # kind:m name:si_value, access: public, file: sys/signal.h, \
                    #                       type: union sigval, struct: %anon64
                    # kind:t name:siginfo_t, type: %anon64, file: sys/signal.h
                    # 
                    # Add the '%anon64' to 'structs'. Add 'si_signo' to the 'struct':'%anon64'
                    # as a member.
                    #
                    if kind == 'm':
                        # Get the type of member this is.
                        key = None
                        for mt in member_types:
                            if mt in rec:
                                key = mt
                                break
                        if key:
                            if key not in rec:
                                raise error.general('field:' + key + \
                                                    ' not in record: ' + self.str_rec(rec))
                            if rec[key] not in self.table[key]:
                                raise error.general('unknown ' + key + ': ' + rec[key] + \
                                                    ': ' + self.str_rec(rec))
                            # Add to the member list.
                            if type(rec) is not dict:
                                output('warning: member rec not a dict: ' + type(rec) +
                                       ': ' + str(rec))
                            # There are duplicated record for the same member
                            # Heap_Block_struct is an example
                            # Thread_Control_struct is another example:
                            # There are multiple tuple like "Thread_Control_struct    ../../cpukit/../../../pc586/lib/include/rtems/score/thread.h    0;"    kind:s"
                            # with different member types
                            memb_exist = False
                            for m in self.table[key][rec[key]]['members']:
                                if rec['name'] == m['name']:
                                    memb_exist = True
                                    break
                            if not memb_exist:
                                self.table[key][rec[key]]['members'].append(rec)
                                if verbose:
                                    output('memb: ' + key + ':' + rec[key] + ': ' + str(rec))
                            else:
                                if verbose:
                                    output('duplicated ' + rec['name'] + ' for ' + rec[key])
                        else:
                            output('warning: unknown m rec: ' + self.str_rec(rec))
                    elif kind in table_keys_ignore:
                        if verbose and False:
                            output('ignore: ' + self.str_rec(rec))
                    else:
                        if name not in self.table[key]:
                            self.table[key][name] = rec
                            if verbose:
                                output('add : ' + key + ':' + name + ': ' + str(rec))
                            # If the key has members create the member's list.
                            if key in member_types:
                                self.table[key][name]['members'] = []
                else:
                    # We do not know this kind of record.
                    if kind not in self.types:
                        output('warning: unknown kind of debug record: ' + self.str_rec(rec))
                        self.types.append(kind)
                    if name not in self.table['unknown']:
                        self.table['unknown'][name] = rec

    def typedef_is(self, rec):
        """Detect the type of typedef in the record."""
        if 'type' not in rec:
            raise error.general('no type field found in typedef: ' + rec['name'])
        t = rec['type']
        if t.startswith('enum'):
            return 'enum'
        elif t.startswith('struct %anon'):
            """To handle the scenario as below:
               Heap_Information_block  ../../cpukit/../../../pc586/lib/include/rtems/score/heap.h      0;"     kind:t  type:struct %anon20
               %anon20 ../../cpukit/../../../pc586/lib/include/rtems/rtems/types.h     0;"     kind:s
            """
            return t.split(' ', 1)[1].strip()
        elif t.startswith('struct'):
            return 'struct'
        elif t.startswith('union'):
            return 'union'
        return t

    def _table_rename(self, match, rep):
        """Find the match text in any key, name or member of any item
        in the symbol table and replace with the rep text."""
        
        def _match_str(left, right):
            """Match a string to be exact. This means the string but also the
            the length."""
            return left == right and len(left) == len(right)
    
        def _update_field(field, match, rep):
            """Update the field if it matches with the replace str."""
            if field.find(match) >= 0:
                return True, field.replace(match, rep)
            return False, field
            
        for key in self.table.keys():
            for name in self.table[key].keys():

                if match == '%anon1_2800' and key == 'function' and name == '_Message_queue_Allocate':
                    print here

                rec = self.table[key][name]
                rec_updated = False
                # if the name matches rename the table entry.
                if _match_str(name, match):
                    del self.table[key][name]
                    name = rep
                    rec_updated = True
                for rf in rec:
                    if rf == 'members':
                        for m in range(0, len(rec[rf])):
                            for field in rec[rf][m]:
                                member_updated, rec[rf][m][field] = \
                                                _update_field(rec[rf][m][field], match, rep)
                            if member_updated:
                                rec_updated = True
                    elif rf == 'args':
                        for a in range(0, len(rec[rf])):
                            member_updated, rec[rf][a] = \
                                            _update_field(rec[rf][a], match, rep)
                            if member_updated:
                                rec_updated = True
                    else:
                        if type(rec[rf]) is str:
                            if _match_str(rec[rf], match):
                                rec_updated, rec[rf] = _update_field(rec[rf], match, rep)
                            elif key == 'typedef':
                                if _match_str(rec[rf], 'struct '+match):
                                    rec_updated, rec[rf] = _update_field(rec[rf], 'struct '+match, rep)
                                elif _match_str(rec[rf], 'union '+match):
                                    rec_updated, rec[rf] = _update_field(rec[rf], 'union '+match, rep)

                        else:
                            output('warning: unsupport field type in rec: ' + str(type(rec[rf])) +
                                   ': ' + str(rec))
                if rec_updated:
                    self.table[key][name] = rec
                
    def clear_anon_struct(self, rec_type):
        """Clear %anon in type of members recursively"""
        for field in rec_type['members']:
            anon = field['type']
            if type(anon) is str:
                #if anon.startswith('%anon2_249'):
                #    output(anon)
                if anon.startswith('%anon'):
                    ispointer = False
                    if anon.endswith('*|'):
                        anon = anon.split(' ', 1)[0]
                        ispointer = True
                    for key in ['struct', 'union', 'function']:
                        if anon in self.table[key]:
                            field['type'] = self.table[key][anon]
                            if ispointer:
                                # Here, 'p' means pointer to struct or union
                                field['type']['kind'] += 'p' 
                            del self.table[key][anon]
                            self.clear_anon_struct(field['type'])
            
                
    def _post_process_table(self):
        """ Post-process the table data we have collected. We need to
        interate over the functions and transforming the argument list
        into a tuple and while doing this transform he enums to the
        enumeration name."""
        # Create the reverse eunms. We need to know what the name of an enum is
        # based on the values of the enum. This is a reverse look up.
        self.table['reverse'] = {}
        self.table['reverse']['enum'] = {}
        for name in self.table['typedef']:
            rec = self.table['typedef'][name]
            t = self.typedef_is(rec)
            if t == 'enum':
                self.table['reverse']['enum'][rec['type']] = rec['name']
        # Create the argument lists from the function decls. We need to break
        # the function down so we can determine the actual types or the anon
        # renames.
        for f in self.table['function']:
            fd = self.table['function'][f]
            # Split the type for the function by the function's name.
            fts = fd['type'].split(fd['name'], 1)
            # Handle the return value first.
            fd['return'] = self._transform_enum(fts[0])
            # Split the arguments into separate parts.
            args = fts[1][1:-1]
            arglist = []
            while True:
                args = self._transform_enum(args)
                asl = args.strip().split(',', 1)
                arglist.append(asl[0])
                if len(asl) <= 1:
                    break;
                args = asl[1].strip()
            fd['args'] = arglist
            self.table['function'][f] = fd
        # At the end rename the anonomous types into actual types.
        for name in self.table['typedef']:
            rec = self.table['typedef'][name]
            t = self.typedef_is(rec)
            if t.startswith('%anon'):
                for key in ['struct', 'union', 'function']:
                    if t in self.table[key]:
                        self._table_rename(t, name)
        for name in self.table['typedef']:
            """
            Following code is to try changing from
                $ cat symbols_union.txt
                cr3: union cr3 { cr3_bits cr3;unsigned int i; };
                $ cat symbols_typedef.txt
                cr3: cr3;
            to
                $ cat symbols_union.txt
                $ cat symbols_typedef.txt
                cr3: union cr3 { cr3_bits cr3;unsigned int i; };
            """
            rec = self.table['typedef'][name]
            td_updated = False
            if name == rec['type']:
                del rec['type']
                if name in self.table['struct']:
                    rec['type'] = self.table['struct'][name]
                    del self.table['struct'][name]
                    td_updated = True
                elif name in self.table['union']:
                    rec['type'] = self.table['union'][name]
                    del self.table['union'][name]
                    td_updated = True
                else:
                    # There are always "void: void", "float: float", "double: double"
                    if name not in ['void', 'float', 'double']:
                        output('warning: ' + name + ' neither is struct nor is union')
                    rec['type'] = name
            if td_updated:
                self.clear_anon_struct(rec['type'])
                
                        
        
#        for name in self.table['typedef']:
#            rec = self.table['typedef'][name]
#            t = self.typedef_is(rec)
#            if t in ['struct', 'union']:
#                anon = self._anon_extract(rec['type'])
#                if anon:
#                    self._table_rename(anon, name)

    def str_rec(self, rec):
        """Return a string the record."""
        if type(rec) is dict and 'kind' in rec and 'name' in rec:
            s = 'kind: %s\tname: %s' % (rec['kind'], rec['name'])
            for f in rec:
                if f not in ['kind', 'name']:
                    if f == 'members':
                        s += ', members:{'
                        delimiter = ''
                        for m in rec[f]:
                            s += delimiter + m['name']
                            delimiter = ', '
                        s += '}'
                    else:
                        s += ', ' + f + ': ' + rec[f]
        else:
            s = str(rec)
        return s
                
    def load(self, objectfile, functions, rawdata = False):
        """Load the debug information as reported by objdump."""
        global verbose
        # See if an object file is provided.
        if objectfile is None:
            raise error.general('no object file provided')        
        # See if objectfile points to a valid file.
        if not os.path.exists(objectfile):
            raise error.general('cannot find objectfile: ' + objectfile)
        # See if a function file is provided. If none is accept all
        # functions.
        self.functions = {}
        if functions is None:
            self.all_functions = True
        else:
            self.all_functions = False
            # Read the function lists.
            lc = 0
            for line in fileinput.input(functions):
                lc += 1
                if line.strip().startswith('#'):
                    continue
                sl = line.strip().split()
                if len(sl) != 2:
                    raise error.general('invalid function: %s:%i' % (functions, lc))
                if sl[0] in self.functions:
                    output('functions: already defined: ' + sl[0])
                else:
                    self.functions[sl[0]] = sl[1]
        if not rawdata:
            # Read the debug information.
            output('Running readelf ....')
            self.exe.set_output(self._readelf_output)
            command.run(self.exe, self.config,
                        (self.env.get('system'), 'readelf_hdr'),
                        [objectfile])
            # Read the debug information.
            self.line = 0
            output('Running objdump ....')
            self.exe.set_output(self._objdump_output)
            command.run(self.exe, self.config,
                        (self.env.get('system'), 'objdump_debug'),
                        [self.arch, self.rtems_version, objectfile])
        else:
            for line in fileinput.input(objectfile):
                self._objdump_output(line)
        self._post_process_table()

    def function_decl(self, function):
        """Return the function as a declaration suitable for compiling
        with a C compiler."""
        decl = ''
        if function in self.table["function"]:
            fd = self.table["function"][function]
            decl = fd['return'] + ' ' + function
            delimiter = '('
            for arg in fd['args']:
                decl += delimiter + arg
                delimiter = ', '
            decl += ')'
        return decl

    def output_anon_type(self, ttype, istypedef):
        s_u = 'struct '
        if ttype['kind'].startswith('u'):
            s_u = 'union '
        if istypedef:
            sdef = s_u + ttype['name'] + ' { '
        else:
            sdef = s_u + ' { '
        for m in ttype['members']:
            if type(m['type']) is str:
                sdef += m['type'] + ' ' + m['name'] + ';'
            else:
                sdef += self.output_anon_type(m['type'], False) + ' ' + m['name'] + ';'
        sdef += ' }'
        if ttype['kind'].endswith('p'):
            sdef += ' * '
        return sdef
        
                    
if command_line:
    cmd_opts = optparse.OptionParser(option_list = cl_options)
    (options, args) = cmd_opts.parse_args()
    if options.version:
        for line in copyright:
            print line
        sys.exit(1)
    try:
        if options.objectfile is None:
            raise error.general('no object file provided, try -o')
        verbose = options.verbose
        if options.quiet:
            command_line = False
        addresses = None
        output(title)
        output()
        obj = symbols(options.config, options.verbose)
        obj.load(options.objectfile, options.functions, options.rawdata)
        output('Functions: ' + str(len(obj.table['function'])))
        output('Structures: ' + str(len(obj.table['struct'])))
        output('Unions: ' + str(len(obj.table['union'])))
        output('Types: ' + str(len(obj.table['typedef'])))
        l = log.log(['symbols_func.txt'])
        if options.showall or options.showfunctions:
            for f in obj.table["function"]:
                if obj.all_functions:
                    output(f + ': ' + obj.function_decl(f) + ';', l)                    
                elif f in obj.functions:
                    output(f + ': ' + obj.function_decl(f) + ';', l)
        del l
        l = log.log(['symbols_struct.txt'])
        if options.showall or options.showstructs:
            for s in obj.table["struct"]:
                sdef = s + ': ' + 'struct ' + s + ' { '
                for m in obj.table["struct"][s]['members']:
                    sdef +=  m['type'] + ' ' + m['name'] + ';'
                output(sdef + ' };', l)
        del l
        l = log.log(['symbols_union.txt'])        
        if options.showall or options.showunions:
            for s in obj.table["union"]:
                sdef = s + ': ' + 'union ' + s + ' { '
                for m in obj.table["union"][s]['members']:
                    sdef +=  m['type'] + ' ' + m['name'] + ';'
                output(sdef + ' };', l)
        del l
        l = log.log(['symbols_typedef.txt'])
        if options.showall or options.showtypes:
            for t in obj.table["typedef"]:
                ttype = obj.table["typedef"][t]['type']
                if type(ttype) is str:
                    output(t + ': ' + ttype + ';', l)
                else:
                    sdef = t + ': ' + obj.output_anon_type(ttype, True) + ';'
                    output(sdef, l)
        del obj
    except error.error, e:
        print e
